{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Notebook for Ex-Post Analysis of Semi-Parametric Estimations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "mk, 16.7.2021"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Packages and Options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "fn_coefs   = \"../results/table_coefficients/df_coefs_lags12.csv\"\n",
    "fn_gen_agg = \"../data/generation_aggregated.csv\"\n",
    "fn_cost    = \"../data/RES_Subsidies_DE_and_UK.xlsx\"\n",
    "fn_out     = \"../results/results_bins_submission.xlsx\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_covs    = \"../results/cov_for_SE/\"\n",
    "dir_figures = \"../figures/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Choose Options**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Choose phi tilde\n",
    "phi_tilde = 0.74\n",
    "\n",
    "# Choose models\n",
    "Models = [\"demperc_12\", \"REperc_12\"]\n",
    "\n",
    "# Dummy for linear specification\n",
    "L = \"yes\" \n",
    "\n",
    "# Start and end date\n",
    "tstart = \"1.1.2015\"\n",
    "tend = \"29.12.2016\"\n",
    "\n",
    "# Define regimes and technologies\n",
    "regimes = [\"low\", \"high\"]\n",
    "res = [\"wind\", \"solar\"]\n",
    "tech = [\"coal\",\"ccgt\"]\n",
    "\n",
    "#  Estimates for Social Cost of Carbon and Levelized Cost of Electricity\n",
    "scc = 50 # €/t\n",
    "lcoe = {'RE': [\"wind\", \"solar\"], 'lcoe': [100,100]} # €/MWh\n",
    "df_lcoe = pd.DataFrame(data=lcoe)\n",
    "\n",
    "# Carbon coefficient per MWh electricity generated\n",
    "d = {'tech': [\"coal\",\"ccgt\"], 'e': [0.89,0.38]} # t CO2 per MWh_el\n",
    "df_e = pd.DataFrame(data=d)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load and prepare data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get estimated coefficients as well as input data used for estimations.\n",
    "df_coefs_in = pd.read_csv(fn_coefs)\n",
    "df_coefs_in = df_coefs_in[df_coefs_in.Model.isin(Models)]\n",
    "df_gen_agg = pd.read_csv(fn_gen_agg, parse_dates=True, index_col=\"date\")[tstart:tend].copy()\n",
    "df_cost = pd.read_excel(fn_cost, sheet_name=\"support\")\n",
    "\n",
    "# Covariance Matrix\n",
    "covs = {}\n",
    "for Model in Models:\n",
    "    for t in tech:\n",
    "        df_cov = pd.read_csv(dir_covs + \"%s_fpn_%s_cov.csv\" % (Model, t))\n",
    "        df_cov = df_cov.set_index(\"index\")\n",
    "        df_cov.index.names = [None] \n",
    "        covs.update( {\"%s_fpn_%s\" % (Model, t) : df_cov} )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare data frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from get_results_ import get_hourly_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>demand</th>\n",
       "      <th>wind</th>\n",
       "      <th>solar</th>\n",
       "      <th>w_wind</th>\n",
       "      <th>w_solar</th>\n",
       "      <th>w_demand</th>\n",
       "      <th>pele_uk</th>\n",
       "      <th>pcarb</th>\n",
       "      <th>subsidy_wind_paid</th>\n",
       "      <th>subsidy_solar_paid</th>\n",
       "      <th>subsidy_wind_mwh</th>\n",
       "      <th>subsidy_solar_mwh</th>\n",
       "      <th>regime</th>\n",
       "      <th>wsubsidy_wind_mwh</th>\n",
       "      <th>wsubsidy_solar_mwh</th>\n",
       "      <th>wpele_wind_mwh</th>\n",
       "      <th>wpele_solar_mwh</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2015-01-01 00:00:00</th>\n",
       "      <td>17791.516667</td>\n",
       "      <td>6989.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000111</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000046</td>\n",
       "      <td>39.61</td>\n",
       "      <td>19.150881</td>\n",
       "      <td>472192.394807</td>\n",
       "      <td>0.0</td>\n",
       "      <td>67.557392</td>\n",
       "      <td>155.19</td>\n",
       "      <td>low</td>\n",
       "      <td>0.020344</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011928</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-01 01:00:00</th>\n",
       "      <td>17489.516667</td>\n",
       "      <td>6971.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000111</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000045</td>\n",
       "      <td>37.18</td>\n",
       "      <td>19.150881</td>\n",
       "      <td>470942.583046</td>\n",
       "      <td>0.0</td>\n",
       "      <td>67.557392</td>\n",
       "      <td>155.19</td>\n",
       "      <td>low</td>\n",
       "      <td>0.020291</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011167</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           demand    wind  solar    w_wind  w_solar  w_demand  \\\n",
       "date                                                                            \n",
       "2015-01-01 00:00:00  17791.516667  6989.5    0.0  0.000111      0.0  0.000046   \n",
       "2015-01-01 01:00:00  17489.516667  6971.0    0.0  0.000111      0.0  0.000045   \n",
       "\n",
       "                     pele_uk      pcarb  subsidy_wind_paid  \\\n",
       "date                                                         \n",
       "2015-01-01 00:00:00    39.61  19.150881      472192.394807   \n",
       "2015-01-01 01:00:00    37.18  19.150881      470942.583046   \n",
       "\n",
       "                     subsidy_solar_paid  subsidy_wind_mwh  subsidy_solar_mwh  \\\n",
       "date                                                                           \n",
       "2015-01-01 00:00:00                 0.0         67.557392             155.19   \n",
       "2015-01-01 01:00:00                 0.0         67.557392             155.19   \n",
       "\n",
       "                    regime  wsubsidy_wind_mwh  wsubsidy_solar_mwh  \\\n",
       "date                                                                \n",
       "2015-01-01 00:00:00    low           0.020344                 0.0   \n",
       "2015-01-01 01:00:00    low           0.020291                 0.0   \n",
       "\n",
       "                     wpele_wind_mwh  wpele_solar_mwh  \n",
       "date                                                  \n",
       "2015-01-01 00:00:00        0.011928              0.0  \n",
       "2015-01-01 01:00:00        0.011167              0.0  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#phi_tilde = 0.74\n",
    "df_dem = get_hourly_df(df_gen_agg, phi_tilde, df_cost, regimes, res)\n",
    "df_dem.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract estimation results and calculate generation offset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_coefs = df_coefs_in.copy()\n",
    "\n",
    "df_coefs = df_coefs[~df_coefs[\"variable\"].str.contains(\"lag\")]\n",
    "df_coefs.tech = df_coefs.tech.map(lambda x: x.split(\"_\")[-1])\n",
    "df_coefs[\"regime\"] = df_coefs.variable.map(lambda x: x.split(\"_\")[0])\n",
    "df_coefs[\"RE\"] = df_coefs.variable.map(lambda x: x.split(\"_\")[1])\n",
    "df_coefs[\"perc\"] = df_coefs.variable.map(lambda x: x.split(\"_\")[-1])\n",
    "\n",
    "df_coefs_ = df_coefs[df_coefs.RE.isin([\"wind\", \"solar\"])].copy()\n",
    "df_coefs_ = df_coefs_.drop(columns = [\"variable\"], axis=1)\n",
    "df_coefs_[\"perc\"] = df_coefs_[\"perc\"].astype(int)\n",
    "\n",
    "df_coefs_[\"perc\"] = df_coefs_[\"perc\"]+1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Aggregate results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tech</th>\n",
       "      <th>RE</th>\n",
       "      <th>Model</th>\n",
       "      <th>regime</th>\n",
       "      <th>deltaQ</th>\n",
       "      <th>deltaQ_se</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ccgt</td>\n",
       "      <td>solar</td>\n",
       "      <td>REperc_12</td>\n",
       "      <td>high</td>\n",
       "      <td>-0.653528</td>\n",
       "      <td>0.010475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>coal</td>\n",
       "      <td>solar</td>\n",
       "      <td>REperc_12</td>\n",
       "      <td>high</td>\n",
       "      <td>-0.346472</td>\n",
       "      <td>0.010475</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tech     RE      Model regime    deltaQ  deltaQ_se\n",
       "0  ccgt  solar  REperc_12   high -0.653528   0.010475\n",
       "1  coal  solar  REperc_12   high -0.346472   0.010475"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nperc = 5\n",
    "df_coefs_agg = df_coefs_.copy()\n",
    "df_coefs_agg[\"std2\"] = df_coefs_agg[\"std\"]*df_coefs_agg[\"std\"]\n",
    "df_coefs_agg = df_coefs_agg.groupby([\"Model\",\"RE\",\"regime\",\"tech\"]).agg({\"coef\":\"mean\", \"std2\": \"sum\"}).reset_index()\n",
    "df_coefs_agg[\"deltaQ\"] = df_coefs_agg.coef\n",
    "df_coefs_agg[\"deltaQ_se\"] = 1/nperc*(df_coefs_agg[\"std2\"])**(1/2)\n",
    "\n",
    "df_Q = df_coefs_agg[[\"tech\", \"RE\", \"Model\", \"regime\", \"deltaQ\", \"deltaQ_se\"]]\n",
    "df_Q.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculate emission offset and cost data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from get_results_ import get_final_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RE</th>\n",
       "      <th>regime</th>\n",
       "      <th>Model</th>\n",
       "      <th>deltaE</th>\n",
       "      <th>deltaE_se</th>\n",
       "      <th>pcarb</th>\n",
       "      <th>market_carbon_rent</th>\n",
       "      <th>market_carbon_rent_se</th>\n",
       "      <th>meb</th>\n",
       "      <th>meb_se</th>\n",
       "      <th>premium</th>\n",
       "      <th>premium_se</th>\n",
       "      <th>market_income</th>\n",
       "      <th>operating_cost</th>\n",
       "      <th>feed_in</th>\n",
       "      <th>premium_paid</th>\n",
       "      <th>premium_paid_t</th>\n",
       "      <th>lcoe</th>\n",
       "      <th>wpele</th>\n",
       "      <th>MAC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>solar</td>\n",
       "      <td>high</td>\n",
       "      <td>REperc_12</td>\n",
       "      <td>-0.556701</td>\n",
       "      <td>0.010137</td>\n",
       "      <td>28.534708</td>\n",
       "      <td>15.885290</td>\n",
       "      <td>0.289244</td>\n",
       "      <td>27.835032</td>\n",
       "      <td>0.506828</td>\n",
       "      <td>11.949742</td>\n",
       "      <td>0.416188</td>\n",
       "      <td>38.349266</td>\n",
       "      <td>22.463976</td>\n",
       "      <td>50.299008</td>\n",
       "      <td>62.577734</td>\n",
       "      <td>112.40823</td>\n",
       "      <td>100</td>\n",
       "      <td>38.349266</td>\n",
       "      <td>110.743061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>solar</td>\n",
       "      <td>high</td>\n",
       "      <td>demperc_12</td>\n",
       "      <td>-0.543815</td>\n",
       "      <td>0.008165</td>\n",
       "      <td>28.534708</td>\n",
       "      <td>15.517605</td>\n",
       "      <td>0.232979</td>\n",
       "      <td>27.190755</td>\n",
       "      <td>0.408238</td>\n",
       "      <td>11.673150</td>\n",
       "      <td>0.335230</td>\n",
       "      <td>38.349266</td>\n",
       "      <td>22.831661</td>\n",
       "      <td>50.022416</td>\n",
       "      <td>62.577734</td>\n",
       "      <td>115.07171</td>\n",
       "      <td>100</td>\n",
       "      <td>38.349266</td>\n",
       "      <td>113.367086</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      RE regime       Model    deltaE  deltaE_se      pcarb  \\\n",
       "0  solar   high   REperc_12 -0.556701   0.010137  28.534708   \n",
       "1  solar   high  demperc_12 -0.543815   0.008165  28.534708   \n",
       "\n",
       "   market_carbon_rent  market_carbon_rent_se        meb    meb_se    premium  \\\n",
       "0           15.885290               0.289244  27.835032  0.506828  11.949742   \n",
       "1           15.517605               0.232979  27.190755  0.408238  11.673150   \n",
       "\n",
       "   premium_se  market_income  operating_cost    feed_in  premium_paid  \\\n",
       "0    0.416188      38.349266       22.463976  50.299008     62.577734   \n",
       "1    0.335230      38.349266       22.831661  50.022416     62.577734   \n",
       "\n",
       "   premium_paid_t  lcoe      wpele         MAC  \n",
       "0       112.40823   100  38.349266  110.743061  \n",
       "1       115.07171   100  38.349266  113.367086  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_deltaE = get_final_results(df_Q, df_dem, df_e, regimes, res, scc, df_lcoe)\n",
    "df_deltaE.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Export"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "wxls = pd.ExcelWriter(fn_out)\n",
    "df_deltaE.melt(id_vars=[\"RE\", \"regime\", \"Model\"]).to_excel(wxls, index=False, sheet_name=\"coefs_Delta_E\")\n",
    "df_Q.melt(id_vars=[\"RE\", \"regime\", \"Model\", \"tech\"]).to_excel(wxls, index=False, sheet_name=\"coefs_Delta_Q\")\n",
    "wxls.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
